<?php
/*
	Revision: 1.0
	Date: 2011-04-29
*/
while(1) {
	set_time_limit(0);
	error_reporting(E_ALL);
	ini_set("display_errors", "Off");
	date_default_timezone_set("PRC");
//	ob_implicit_flush();
//	ob_end_flush();

	$c = new keywordSpider($dbhost="localhost", $dbuser="root", $dbpw="", $dbname = "happy", $pconnect = 0); //此处可指定数据库连接参数
	//获取用户
	$userInfo = $c->getUid();
	$uid = $userInfo['0']['uid'];
     //URL过滤数组
	$urlArrayData = $c->getUrlFliter($uid);
	if(is_array($urlArrayData) && !empty($urlArrayData)) {
		foreach ($urlArrayData as $keyUrlP=>$valueUrlP){
			$urlArray[] = $valueUrlP['url'];
		}
	}
	$filter["urls"] = $urlArray;
	//过滤关键词
	$worldArrayData = $c->getWorldFliter($uid);
	if(is_array($worldArrayData) && !empty($worldArrayData)) {
		foreach ($worldArrayData as $keyWorldP=>$valueWorldP){
			$worldArray[] = $valueWorldP['world'];
		}
	}
	$filter["keywords"] = $worldArray; //关键字过滤数组

	$worldKeyArrayData = $c->getWorldKey($uid);
	if(is_array($worldKeyArrayData) && !empty($worldKeyArrayData)) {
		foreach ($worldKeyArrayData as $keyWorldKeyP=>$valueWorldKeyP){
			/*
				fetch($uid,$keyword,$source,$filter) 采集指定搜索引擎关键字入库,符合URL过滤数组或关键字过滤数组任何一个条件则不入库
				$uid:用户的ID
				$keyword:关键字
				$source:来源 目前只支持 baidu 1 google 2  soso 3
				$fliter:过滤器 fliter["urls"] URL过滤数组    fliter["keywords"]数组 关键字过滤数组
			*/
			$c->fetch($uid,$valueWorldKeyP['words'],"baidu",$filter,$valueWorldKeyP['id']);
			sleep(4);
			$c->fetch($uid,$valueWorldKeyP['words'],"google",$filter,$valueWorldKeyP['id']);
			sleep(4);
			$c->fetch($uid,$valueWorldKeyP['words'],"soso",$filter,$valueWorldKeyP['id']);
			sleep(4);
		}
	}
	$c->userInfoTime($uid);
}



























































class keywordSpider //关键字spider
{
	var $bug,$dblink;

	function __construct($dbhost="localhost", $dbuser="root", $dbpw="", $dbname = "test", $pconnect = 0){
		$dblink = new db_sql;
		$dblink->connect($dbhost, $dbuser, $dbpw, $dbname, $pconnect);
		$this->dblink = $dblink;
		$this->dblink->query("CREATE TABLE if not exists  `core_keywordspider` (
                 `id` int(11) NOT NULL AUTO_INCREMENT,
                 `keyword` varchar(255) CHARACTER SET utf8 DEFAULT NULL COMMENT '关键字',
                 `title` varchar(255) CHARACTER SET utf8 DEFAULT NULL COMMENT '链接标题',
                 `url` varchar(255) CHARACTER SET utf8 DEFAULT NULL COMMENT '链接地址',
                 `uid` int(11) DEFAULT NULL COMMENT '用户ID',
                 `summary` text CHARACTER SET utf8 COMMENT '摘要',
                 `source` varchar(255) CHARACTER SET utf8 DEFAULT NULL COMMENT '来源 baidu 1 google 2 soso 3',
                 PRIMARY KEY (`id`)
               ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ");
		$this->bug = 1;
		$this->maxpage = 3; //最大采集分页数
		$this->sleeptime = 3; //每分页采集间隔时间
	}

    /*
	 $uid:用户的ID
	 */
	 function getUrlFliter($uid) {
		 $sql = "SELECT *  FROM `core_keywordurl` WHERE `uid` = ".$uid;
		 $query = $this->dblink->query($sql);
		 return $this->dblink->fetch_array($query);
	 }

	/*
	 $uid:用户的ID
	 */
	 function getWorldFliter($uid) {
		 $resultData = array();
		 $sql = "SELECT *  FROM `core_keywordfilter` WHERE `uid` = ".$uid;
		 $query = $this->dblink->query($sql);
		 while( $row = $this->dblink->fetch_array($query)){
				$resultData[]=$row;
		 }
		 return $resultData;
	 }

	 /*
	  $uid:用户ID
	  */
	  function getWorldKey($uid) {
		 $resultData = array();
		 $sql = "SELECT *  FROM `core_keyword` WHERE `uid` = ".$uid;
		 $query = $this->dblink->query($sql);
		 while( $row = $this->dblink->fetch_array($query)){
				$resultData[]=$row;
		}
		 return $resultData;
	  }

	  /*
	   return 获取最后时间，进行更新
	   */
	   function getUid() {
		   $resultData = array();
		   $sql = "SELECT * FROM `core_user` ORDER BY `core_user`.`worldtime` ASC LIMIT 0 , 1 ";
		   $query = $this->dblink->query($sql);
		   	while( $row = $this->dblink->fetch_array($query)){
				$resultData[]=$row;
			}
			return $resultData;

	   }

	   /*
	    更新时间
	    */
		function userInfoTime($uid) {
		   $worldTime = time();
		   $sql = "UPDATE `happy`.`core_user` SET `worldtime` = ".$worldTime." WHERE `core_user`.`uid` =".$uid." LIMIT 1 ;";
		   $query = $this->dblink->query($sql);
		}
/*
	$uid:用户的ID
	$keyword:关键字
	$source:来源
	$fliter:过滤器 fliter["urls"] URL过滤数组    fliter["keywords"]数组 关键字过滤数组
*/
	function fetch($uid,$keyword,$source,$filter,$id){
		switch ($source){
			case "baidu":
				$this->baiduFetch($uid,$keyword,$filter,$id);
				break;
			case "google":
				$this->googleFetch($uid,$keyword,$filter,$id);
				break;
			case "soso":
				$this->sosoFetch($uid,$keyword,$filter,$id);
				break;
			default:
				die("来源错误");
		}
	}

	function baiduFetch($uid,$keyword,$filter,$id,$page=0){
		if($page < $this->maxpage){
			$url = "http://www.baidu.com/s?wd=".urlencode($keyword)."&pn=".($page*10)."&usm=1";
			$str = $this->HappyCj('',$url,$url);
			$str = $this->convertEncode('gb2312', 'UTF-8', $str);
			$baiduKeywordResultA =  $this->pregmessage($str, '<table cellpadding="0" cellspacing="0" class="result"[list]</table>', 'list', -1);
			foreach($baiduKeywordResultA as $baiduKeywordResult ){
				$addKeyword = $keyword; //关键字

				$addTitle = $this->pregmessage($baiduKeywordResult, '<h3 class="t">[list]</h3>', 'list'); //标题
				$addTitle = strip_tags($addTitle[0]);
				if(!$addTitle) continue;

				$addUrl = $this->pregmessage($baiduKeywordResult, 'href="[list]" target="_blank"', 'list'); //URL
				$addUrl = $addUrl[0];
				if(!$addUrl) continue;

				$addSummary = $this->pregmessage($baiduKeywordResult, '<font size=-1>[list]<br>', 'list'); //摘要
				$addSummary = strip_tags($addSummary[0]);
				if(!$addSummary) continue;


				$this->addResult($uid,$filter,$addKeyword,$addTitle,$addUrl,$addSummary,$id,1);
			}
			sleep($this->sleeptime);
			$this->baiduFetch($uid,$keyword,$filter,$id,$page+1);
		}
		//var_dump(func_get_args());die();
	}


	function googleFetch($uid,$keyword,$filter,$id,$page=0){
		if($page < $this->maxpage){
			$url = "http://www.google.com.hk/search?hl=zh-CN&q=".urlencode($keyword)."&start=".($page*10);
			$str = $this->googleCurl('',$url,$url);
			$googleKeywordResultA =  $this->pregmessage($str, '<!--m-->[list]<!--n-->', 'list', -1);
			foreach($googleKeywordResultA as $googleKeywordResult ){
				$addKeyword = $keyword; //关键字
				$addTitle = $this->pregmessage($googleKeywordResult, '<h3 class="r">[list]</h3>', 'list'); //标题
				$addTitle = strip_tags($addTitle[0]);
				if(!$addTitle) continue;

				$addUrl = $this->pregmessage($googleKeywordResult, '<a href="[list]" target=_', 'list'); //URL
				$addUrl = $addUrl[0];
				if(!$addUrl) continue;

				$addSummary = $this->pregmessage($googleKeywordResult, '<div class="s">[list]<br>', 'list'); //摘要
				$addSummary = strip_tags($addSummary[0]);
				if(!$addSummary) continue;
				$this->addResult($uid,$filter,$addKeyword,$addTitle,$addUrl,$addSummary,$id,2);
			}
			sleep($this->sleeptime);
			$this->googleFetch($uid,$keyword,$filter,$id,$page+1);
		}
	}

	function sosoFetch($uid,$keyword,$filter,$id,$page=0){
		if($page < $this->maxpage){
			$url = "http://www.soso.com/q?w=".urlencode($keyword)."&lr=&sc=web&ch=w.p&num=10&gid=&cin=&site=&sf=0&sd=0&pg=".($page+1);
			$str = $this->HappyCj('',$url,$url);
			$str = $this->convertEncode('gb2312', 'UTF-8', $str);
			$sosoKeywordResultA =  $this->pregmessage($str, '<li loc="[list]</li>', 'list', -1);
			foreach($sosoKeywordResultA as $sosoKeywordResult ){
				$addKeyword = $keyword; //关键字

				$addTitle = $this->pregmessage($sosoKeywordResult, '<h3>[list]</h3>', 'list'); //标题
				$addTitle = strip_tags($addTitle[0]);
				if(!$addTitle) continue;

				$addUrl = $this->pregmessage($sosoKeywordResult, 'href="[list]" id=', 'list'); //URL
				$addUrl = $addUrl[0];
				if(!$addUrl) continue;

				$addSummary = $this->pregmessage($sosoKeywordResult, '<p class="ds">[list]</p>', 'list'); //摘要
				$addSummary = strip_tags($addSummary[0]);
				if(!$addSummary) continue;

				if(empty($id)) {
					exit('这出错了，是0');
				}
				$this->addResult($uid,$filter,$addKeyword,$addTitle,$addUrl,$addSummary,$id,3);
			}
			sleep($this->sleeptime);
			$this->sosoFetch($uid,$keyword,$filter,$id,$page+1);
		}
	}

	function addResult($uid,$filter,$addKeyword,$addTitle,$addUrl,$addSummary,$id,$addSource){
		$addKeyword = mysql_escape_string($addKeyword);
		$addTitle = mysql_escape_string($addTitle);
		$addUrl = mysql_escape_string($addUrl);
		$addSummary = mysql_escape_string($addSummary);
		$addSource = mysql_escape_string($addSource);
		if((isset($filter["keywords"]) && !stripos($addTitle,$filter["keywords"])) && (isset($filter["urls"]) && !in_array($addUrl,$filter["urls"])) ){
			$query = $this->dblink->query("SELECT * FROM core_keywordspider WHERE url='{$addUrl}' and source='{$addSource}' and uid='{$uid}'");
			$isExistUser = $this->dblink->fetch_array($query);
			if (!$isExistUser) {
				$sql = "insert into core_keywordspider ( keyword, title, url, uid, summary, source ,worldid) values ( '{$addKeyword}', '{$addTitle}', '{$addUrl}', {$uid}, '{$addSummary}', '{$addSource}', '{$id}')";
				$this->dblink->query($sql);
				//echo "{$addSource}\t<a href={$addUrl}>{$addTitle}</a>\t入库成功<br>";
			}else{
				//echo "{$addSource}\t<a href={$addUrl}>{$addTitle}</a>\t<font color=red>数据库已存在</font><br>";
			}
		}
	}

	function HappyCj($cookies,$referer,$url){
		$context=array('http' => array ('header'=> 'Range: bytes=0-', ),);
		$xcontext = stream_context_create($context);
		$result=file_get_contents($url,FALSE,$xcontext);
		return $result;
	}

	function HappyCjCurl($cookies,$referer,$url){
		$c = curl_init();
		curl_setopt($c, CURLOPT_URL, $url);
		curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
		//curl_setopt($c, CURLOPT_HEADER, $head);
		//curl_setopt($c, CURLOPT_PROXY, $proxy);
		//curl_setopt($c, CURLOPT_POST, 1);
		//curl_setopt($c, CURLOPT_POSTFIELDS, $post);
		curl_setopt($c, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; CIBA; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)");
		curl_setopt($c, CURLOPT_REFERER, $referer);
		curl_setopt($c, CURLOPT_COOKIE, $cookies);
		curl_setopt($c, CURLOPT_TIMEOUT, 0);
		$result = curl_exec($c);
		curl_close($c);
		return $result;
	}

	function googleCurl($cookies,$referer,$url) {
		$headers = array (
			'User-Agent:Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.9) Gecko/20100315 Firefox/3.5.9',
			'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
			'Accept-Language: zh-cn,zh;q=0.5',
			'Accept-Charset: GB2312,utf-8;q=0.7,*;q=0.7',
			'Keep-Alive:300',
			'Connection:keep-alive',
			'Cookie: PREF=ID=dd8f03ed1eafb98f:U=97879ad4b332d39b:FF=1:LD=zh-CN:NW=1:TM=1285129421:LM=1285129424:S=yzHbn6gX-QahTuwc; NID=38=BcPsmEvi1byDr1mZSVXH9gnx800dmcaUH76T7BkhDnJQ3L0jGOi_SR8xFhNxuti35wgoJAWbTNxd3wsGAqhTlkI0SB1BbtbBZhDPvcKpExthBllQ61FX4iVwL2d-L0ob'
			);
		$ch = curl_init();
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
		curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
		$newfile = curl_exec($ch);
		return $newfile;
	}

	function convertEncode($in, $out, $string)//convertEncode('UTF-8','gb2312',$content);
	{
		if (function_exists('mb_convert_encoding')) {
			return mb_convert_encoding($string, $out, $in);
		} elseif (function_exists('iconv')) {
			return iconv($in, $out, $string);
		} else {
			return $string;
		}
	}

	/**
	 * 解析内容
	 * pregmessage($sourcehtml, '<div class="pd10">[list]</table>', 'list');	//解析列表区域
	 * pregmessage($subjecturlarr[0], sstripslashes(trim('href="[url]" target')), 'url', -1);		//解析上步过虑后的结果
	 */
	function pregmessage($message, $rule, $getstr, $limit=1) {
		$result = array('0'=>'');
		$rule = $this->convertrule($rule);		//转义正则表达式特殊字符串
		$rule = str_replace('\['.$getstr.'\]', '\s*(.+?)\s*', $rule);	//解析为正则表达式
		if($limit == 1) {
			preg_match("/$rule/is", $message, $rarr);
			if(!empty($rarr[1])) {
				$result[0] = $rarr[1];
			}
		} else {
			preg_match_all("/$rule/is", $message, $rarr);
			if(!empty($rarr[1])) {
				$result = $rarr[1];
			}
		}
		return $result;
	}

	/**
	 * 转义正则表达式字符串
	 */
	function convertrule($rule) {
		$rule = preg_quote($rule, "/");		//转义正则表达式
		$rule = str_replace('\*', '.*?', $rule);
		$rule = str_replace('\|', '|', $rule);
		return $rule;
	}


	//注册用户开始
	function reg_usr($user){
		$user = str_replace("'",'"',$user);
		$query = $this->dblink->query("SELECT * FROM {$this->tablepre}member WHERE username='{$user}'");
		$isExistUser = $this->dblink->fetch_array($query);
		if (!$isExistUser) {
			$cyask_user = $user;
			$cyask_pw = md5($user . "yingying");
			$cyask_email = rand_mail();
			$onlineip = rand_ip();
			$timestamp = time() - mt_rand(86400, 31536000);
			$bday = rand_birthday();
			$gender = mt_rand(0, 2);
			$qq = rand_code2(mt_rand(7, 10), $type = 1);

			$sql = "INSERT INTO {$this->tablepre}member SET username='$cyask_user',password='$cyask_pw',qq='$qq',bday='$bday',gender='$gender',email='$cyask_email',adminid='5',regip='$onlineip',lastlogin='$timestamp'";
			if ($query = $this->dblink->query($sql)) {
				$cyask_uid = $this->dblink->insert_id();
			}
			update_score($cyask_uid, $score_register, '+');
			$echoStr = "添加用户：{$user}<br>\r\n";
		}else{
			$cyask_uid = $isExistUser["uid"];
			$echoStr = "用户：{$user}已存在<br>\r\n";
		}
		if ($this->bug)
				echo $echoStr;
		return $cyask_uid;
	}
	//注册用户结束
}

class db_sql
{
	var $querynum = 0;
	var $isdebug = 0;

	function connect($dbhost, $dbuser, $dbpw, $dbname = '', $pconnect = 0)
	{
		if($pconnect)
		{
			if(!@mysql_pconnect($dbhost, $dbuser, $dbpw))
			{
				$this->halt('Can not connect to MySQL server');
			}
		}
		else
		{
			if(!@mysql_connect($dbhost, $dbuser, $dbpw))
			{
				$this->halt('Can not connect to MySQL server');
			}
		}

		if($this->version() > '4.1')
		{
			$charset = 'utf-8';		// 程序默认字符集
			$dbcharset = 'utf8';	// MySQL 字符集
			if(!$dbcharset && in_array(strtolower($charset), array('gbk', 'utf-8')))
			{
				$dbcharset = str_replace('-', '', $charset);
			}

			if($dbcharset)
			{
				mysql_query("SET character_set_connection=$dbcharset, character_set_results=$dbcharset, character_set_client=binary");
			}

			if($this->version() > '5.0.1')
			{
				mysql_query("SET sql_mode=''");
			}
		}

		if($dbname)
		{
			mysql_select_db($dbname);
		}

	}

	function select_db($dbname)
	{
		return mysql_select_db($dbname);
	}

	function query($sql, $type = '')
	{
		global $debug, $cyask_starttime, $sqldebug;
		if ($this->isdebug) echo $sql."<br>";
		$func = $type == 'UNBUFFERED' && @function_exists('mysql_unbuffered_query') ? 'mysql_unbuffered_query' : 'mysql_query';
		if(!($query = $func($sql)) && $type != 'SILENT')
		{
			$this->halt(mysql_error(), $sql);
		}
		$this->querynum++;
		return $query;
	}

	function fetch_array($query)
	{
		return mysql_fetch_assoc($query);
	}

	function affected_rows()
	{
		return mysql_affected_rows();
	}

	function error()
	{
		return mysql_error();
	}

	function errno()
	{
		return intval(mysql_errno());
	}

	function result($query, $row)
	{
		$query = @mysql_result($query, $row);
		return $query;
	}

	function num_rows($query)
	{
		$query = mysql_num_rows($query);
		return $query;
	}

	function num_fields($query)
	{
		return mysql_num_fields($query);
	}

	function free_result($query)
	{
		mysql_free_result($query);
	}

	function insert_id()
	{
		$id = mysql_insert_id();
		return $id;
	}

	function fetch_row($query)
	{
		$query = mysql_fetch_row($query);
		return $query;
	}

	function fetch_fields($query)
	{
		return mysql_fetch_field($query);
	}

	function version()
	{
		return mysql_get_server_info();
	}

	function close()
	{
		return mysql_close();
	}

	function halt($message = '', $sql = '')
	{
		echo "message:{$message}<br>sql:{$sql}";die;
	}
}




?>